import datetime

import requests
from bs4 import BeautifulSoup
import random
import os
import json
import pymysql

# 收集到的常用Header
my_headers = [
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
    'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
    'Opera/9.25 (Windows NT 5.1; U; en)',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
    'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
    'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
    'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
    "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 "
]

'''
数据连接参数
'''
host = "ring1"
port = 3306
db = "movie_analysis"
user = 'root'
password = 'root'

'''
数据库连接
'''
def get_connect():
    connect = pymysql.connect(host=host, port=port, db=db, user=user, password=password, use_unicode='utf-8')
    return connect


'''
    获取电影信息
    1. 电影名称
    2. 电影封面图片
    3. 详情页 URL（并从中提取出 id)
    4. 获取 id
    5. 电影详细信息
'''


def get_main(url: str, f):
    # 插入新的电影
    insert_new = 'insert into movies value (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'

    # 新纪录插入
    insert_add = 'insert into movie_add value (%s, %s)'

    # 更新信息
    update = 'update movies set rating_count=%s, rating=%s where id=%s'

    # 插入修改信息
    insert_modify = 'insert into movie_modify value (%s, %s, %s, %s, %s, %s)'

    # 检查 修改 是否已经修改
    select_modify = 'select *  from movie_modify where date=%s and movie_id=%s'

    # 检查 是否已经 添加
    select_add = 'select *  from movie_modify where date=%s and movie_id=%s'

    # 检查是否有此相关数据
    select = 'select * from movies where id=%s'

    # 昨天
    today = datetime.date.today()
    oneday = datetime.timedelta(days=1)
    yesterday = today - oneday

    # 选择 header 信息
    headers = {'User-Agent': random.choice(my_headers)}
    # 发送请求，接收 html
    response = requests.get(url=url, headers=headers)

    # 请求返回 json 数据
    subjects = response.json()['subjects']

    # 每一部电影信息
    for movie in subjects:
        name = movie['title']
        cover = movie['cover']
        url = movie['url']
        movie_id = movie['id']
        try:

            # 获取数据库连接
            connect = get_connect()

            # 获取 cursor
            cursor = connect.cursor()

            count_modify = cursor.execute(select_modify, (yesterday, movie_id))
            count_add = cursor.execute(select_add, (yesterday, movie_id))

            # 信息已修改过或添加
            if count_modify + count_add:
                print("电影 " + name + " 信息今日已更新或添加")

            else:
                # 获取详细信息
                res = get_message(movie_id)

                # 判断是否已有此电影
                count = cursor.execute(select, movie_id)
                result = cursor.fetchone()

                if count:
                    # 数据库中已有此数据
                    pre_rating_count = result[10]
                    pre_rating = result[11]
                    new_rating_count = res['rating_count']
                    new_rating = res['rating']

                    if (pre_rating_count != new_rating_count) or (pre_rating != new_rating):

                        cursor.execute(update, (new_rating_count,
                                                new_rating,
                                                movie_id))
                        connect.commit()

                        cursor.execute(insert_modify, (movie_id,
                                                       pre_rating_count,
                                                       pre_rating,
                                                       new_rating_count,
                                                       new_rating,
                                                       yesterday))
                        connect.commit()

                        print("修改 " + name + " 的信息")
                    else:
                        print("电影 " + name + " 信息昨日未更新")

                else:
                    # 往数据库中添加新的数据
                    cursor.execute(insert_new, (movie_id,
                                                name,
                                                cover,
                                                url,
                                                res['movie_type'],
                                                res['director'],
                                                res['author'],
                                                res['actor'],
                                                res['date_published'],
                                                res['description'],
                                                res['rating_count'],
                                                res['rating']))
                    connect.commit()

                    # 往 sortByType 中添加数据
                    for type in res['movie_type'].split("/"):
                        cursor.execute("insert into sortByType value (%s, %s)", (movie_id, type))
                        connect.commit()

                    cursor.execute(insert_add, (movie_id, yesterday))
                    connect.commit()

                    print("添加 " + name + " 的信息")

        except Exception as e:
            print(e)
            f.write(name + '\t' + movie_id + '\t' + str(e) + '\n')
            f.flush()
            print("电影 " + name + " 操作失败")


# 获取电影详细信息
def get_message(movie_id):
    url = 'https://movie.douban.com/subject/{}/'

    '''
    似乎 pymmsql 不支持条件插入
    '''
    # # 数据插入语句
    # insert = "insert into 'movies'('type', 'director', 'author', 'actor', 'date_published', 'description', " \
    #          "'rating_count', 'rating') value (%s, %s, %s, %s, %s, %s, %s, %s) where 'id' = %s"

    res = {}

    headers = {'User-Agent': random.choice(my_headers)}
    # 发送请求，接收 html
    response = requests.get(url=url.format(str(movie_id)), headers=headers).content.decode()

    # BeautifulSoup 解析返回网页
    detail = BeautifulSoup(response, 'html.parser')

    # print(detail)

    # 获取类型信息
    info = detail.find_all(name='span', attrs={'property': 'v:genre'})
    movie_type = info[0].text
    for t in info[1:]:
        movie_type = movie_type + '/' + t.text
    # print("电影类型：" + movie_type)
    res['movie_type'] = movie_type

    # 导演信息
    info = detail.find_all(name='a', attrs={'rel': 'v:directedBy'})
    director = info[0].text
    for t in info[1:]:
        director = director + '/' + t.text
    # print("导演：" + director)
    res['director'] = director

    # 编剧信息
    # 编剧信息无特殊属性，通过导演节点进行查找
    '''
    <span>
        <span class="pl">导演</span>: 
        <span class="attrs">
            <a href="/celebrity/1274313/" rel="v:directedBy">邱礼涛</a>
        </span>
    </span>
    <br/>
    <span>
        <span class="pl">编剧</span>: 
        <span class="attrs">
            <a href="/celebrity/1274313/">邱礼涛</a> 
        </span>
    </span>
    <br/>
    '''
    info = info[0].parent.parent.next_sibling.next_sibling.next_sibling
    info = info.find_all(name='a')
    # print(info)
    author = info[0].text
    for t in info[1:]:
        author = author + '/' + t.text
    # print("编剧：" + author)
    res['author'] = author

    # 演员信息
    info = detail.find_all(name='a', attrs={'rel': 'v:starring'})
    actor = info[0].text
    for t in info[1:10]:
        actor = actor + '/' + t.text
    # print("演员：" + actor)
    res['actor'] = actor

    movie_message = detail.find(name='script', attrs={"type": "application/ld+json"})
    info = json.loads(str(movie_message.contents[0]), strict=False)

    # 上映日期
    date_published = info['datePublished']
    # print('上映日期：' + date_published)
    res['date_published'] = date_published

    # 描述
    description = info['description']
    # print('简介：' + description)
    res['description'] = description

    # 参与评分人数
    rating_count = info['aggregateRating']['ratingCount']
    # print("评分人数：" + rating_count)
    res['rating_count'] = rating_count

    # 评分
    rating = info['aggregateRating']['ratingValue']
    # print("评分：" + rating)
    res['rating'] = rating

    # print(res)
    return res

    # # 数据存入数据
    # connect = get_connect()
    # cursor = connect.cursor()
    # cursor.execute(insert, (movie_type, director, author, actor, date_published, description,
    #                         rating_count, rating, movie_id))
    # connect.commit()


if __name__ == '__main__':
    sorts = ["热门", "最新", "经典", "可播放", "豆瓣高分", "冷门佳作", "华语", "欧美", "韩国", "日本", "动作", "喜剧", "爱情", "科幻", "悬疑", "恐怖", "成长"]

    movies_sorts_url = "https://movie.douban.com/j/search_subjects?type=movie&tag={}&sort=recommend&page_limit=1000&page_start=0"

    f = open('./' + str(datetime.date.today()) + '_error.txt', 'a', encoding='utf-8')

    for sort in sorts:
        # 存储 Exception
        get_main(movies_sorts_url.format(sort), f)

        print("finished " + sort + "类型")

    # get_main("https://movie.douban.com/j/search_subjects?type=movie&tag=华语&sort=recommend&page_limit=20&page_start=0")
    #
    # res = get_message('1292063')
    # print(res)
